const inintDb = require('../utils/initDb_cnblog')();
const cnblog = require('../models/cnblogs_message');
const Crawler = require("crawler");

var index = 1;
setTimeout(function run() {    
    var c = new Crawler({
        maxConnections: 10,
        // This will be called for each crawled page
        callback: function (error, res, done) {
            if (error) {
                console.log(error);
            } else {
                var $ = res.$;

                var list = $('#main #post_list .post_item .post_item_body').toArray();
                list.forEach(element => {
                    var title = $(element).find('h3 a').text();
                    var author = $(element).find('.post_item_foot .lightblue').text();
                    var ReleaseTime = $(element).find('.post_item_foot').text().match(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}/).toString();
                    cnblog.create({ title: title, author: author, ReleaseTime: ReleaseTime });
                });
            }
        }
    })
    var URL = 'https://www.cnblogs.com/sitehome/p/' + index;
    c.queue(URL);
    if (index < 200) {
        setTimeout(run, 1)
        index++;
    }else{
        console.log('done');
    };
}, 1);